In this notebook, a template is provided for you to implement your functionality in stages which is required to successfully complete this project. If additional code is required that cannot be included in the notebook, be sure that the Python code is successfully imported and included in your submission, if necessary. Sections that begin with 'Implementation' in the header indicate where you should begin your implementation for your project. Note that some sections of implementation are optional, and will be marked with 'Optional' in the header.
In addition to implementing code, there will be questions that you must answer which relate to the project and your implementation. Each section where you will answer a question is preceded by a 'Question' header. Carefully read each question and provide thorough answers in the following text boxes that begin with 'Answer:'. Your project submission will be evaluated based on your answers to each of the questions and the implementation you provide.
Note: Code and Markdown cells can be executed using the Shift + Enter keyboard shortcut. In addition, Markdown cells can be edited by typically double-clicking the cell to enter edit mode.
import pickle
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as pimg
import glob
import cv2
from datetime import timedelta
from time import time
from random import random
import os.path
%matplotlib inline
# Everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
########################################################
# Helper function to plot images side-by-side with title
########################################################
def plot_gallery(images, titles, h, w, n_row=5, n_col=4):
"""Helper function to plot a gallery of portraits"""
# plt.figure(figsize=(2.0 * n_col, 2.4 * n_row))
plt.figure(figsize=(3.0 * n_col, 3.6 * n_row))
plt.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.25)
llen = 11
for i in range(min(n_row * n_col, len(images))):
plt.subplot(n_row, n_col, i + 1)
plt.imshow(images[i], cmap=plt.cm.gray)
title_i = titles[i]
if len(title_i) >= llen:
title_i = titles[i][llen:]
plt.title(title_i, size=8)
plt.xticks(())
plt.yticks(())
#####################################################################
# class to keep track the characteristics of each Lane line detection
# N.B.
# First Implementation, use this class just as global data struct
# Future TODO: refactor code towards OOP, add methods and classes
#
# This class is vital to reject bad sample points and fitted line
#
# Initial Version: have ZERO class parameters!
# Set it with short memory of 10 last entries.
#
#####################################################################
class Laneline():
def __init__(self):
# cache size, tunable:
self.n = 5
# xbase shift limit (unit: pixel) for rejection
self.xbase_offlimit = 100 # 300
# quadratic coefficient c2 (x-axis intercept) offlimit to reject!
# c2 is x-axis intercept of fitted curve line, set as ~5% of Xmax
self.c2fit_offlimit = 100
# x base values of the last n fits of the line
# append use: list_name.append(value)
# to pop use: list_name.pop(0)
self.all_xbases = []
#average x base of the fitted line over the last n iterations
# self.mean_xbase = int(np.mean(self.all_xbases))
self.mean_xbase = None
# quadratic polynomial coefficients of the last n lines fits
# e.g. [ array([ 0.00275482, 0.03030303, 0.33333333]),
# array([ 0.00275532, 0.03030633, 0.33333465]) ]
self.all_fits = []
# polynomial coefficient averaged over the last n iterations
# self.mean_fit = np.mean(self.all_fits, axis=0)
# e.g. array([ 0.00275482, 0.03030303, 0.33333333])
self.mean_fit = None
# X/Y coordinates of last found points from sliding windows
self.lastx = None
self.lasty = None
###################################
# Undefined -or- obsoleted #
# Some of these can be used to #
# enhance tracking in the future #
###################################
# was the line detected in the last iteration?
# If not, then rejected it and use prior cache
# N.B. before n good ones cached will not tell
# self.detected = False
# x top (where y=0) values of last n fit line
# x top is calculated post polynomial fit y=0
# it is good scalar to tell if fit an outlier
# Initially not used, add support when needed
# self.all_xtops = []
#average x top of the fitted line over the last n iterations
# self.mean_xtop = int(np.mean(self.mean_xtop))
# Initially not used, add support when needed
# self.mean_xtop = None
# This is obsoleted, as it is the last one in list: all_fits
# polynomial coefficients for the most recent fit
# self.current_fit = [np.array([False])]
# obsolete: last_xbase = all_xbases[-1]
# self.last_xbase = None
# radius of curvature of the fitted line in some units
# self.radius_of_curvature = None
# distance in centimeters from vehicle center to lane center
# self.line_base_pos = None
# difference in fit coefficients between last and new fits
# self.diffs = np.array([0,0,0], dtype='float')
# x values for detected line pixels
# self.allx = None
# y values for detected line pixels
# self.ally = None
# class method to tell if new_xbase is valid one or not
# to reject noise, it also updates cache if appropriate
def xbase_valid(self, new_xbase):
if len(self.all_xbases) < self.n:
# just add, if not enough history
self.all_xbases.append(new_xbase)
self.mean_xbase = int(np.mean(self.all_xbases))
return True
else:
# when this class instance has cached enough hist. records
# if abs(new_xbase - self.mean_xbase) > self.xbase_offlimit:
if abs(new_xbase - self.all_xbases[-1]) > self.xbase_offlimit:
# when new_xbase is offlimit, user should call:
# xbase_get() to get last good xbase from cache
return False
else:
# update cache and mean
self.all_xbases.pop(0)
self.all_xbases.append(new_xbase)
self.mean_xbase = int(np.mean(self.all_xbases))
return True
# class method for user to get last good xbase in cache
def xbase_get(self):
if len(self.all_xbases):
return self.all_xbases[-1]
# class method to tell if a new fit is valid one or not
# to reject noise, it also updates cache if appropriate
# N.B. Input: new_fit is np.array return of np.polyfit()
def fit_valid(self, new_fit):
if len(self.all_fits) < self.n:
# just add, if not enough history
self.all_fits.append(new_fit)
self.mean_fit = np.mean(self.all_fits, axis=0)
return True
else:
# when this class instance has cached enough fits history
if ( abs(new_fit[2] - self.mean_fit[2]) > self.c2fit_offlimit ):
# when new_fit X-intercept is offlimit, user should call:
# fit_get() to get last good fit (coeeficient) from cache
return False
else:
# update cache and mean
self.all_fits.pop(0)
self.all_fits.append(new_fit)
self.mean_fit = np.mean(self.all_fits, axis=0)
return True
# class method for user to get last good fit coefficients
def fit_get(self):
if len(self.all_fits):
return self.mean_fit
#return self.all_fits[-1]
#################################################################
# Program globals (most of calibration / testing images specific)
#################################################################
# Camera(Video) is set with the same size for all Images/Frames
# N.B. there are a few images misformed, e.g. calibration7 & 15
# We should normalize image size whenever necessary
w = 1280
h = 720
imsize = (w, h)
# Output file base path (relative to this IPython notebook file)
outputpath = 'output_images/'
#######################################################
# Calibration section (most not used in video pipeline)
#######################################################
# N.B. Not all are used in image pipeline here, most are used
# to generate results (camera matrix) to be used in pipeline!
# First
# Prepare static object points of whole chessboard, like
# (0,0,0), (1,0,0), ...,(8,5,0) left-2-right, top-2-down
objp = np.zeros((6*9,3), np.float32)
objp[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1,2)
# N.B. not all objectpoints from every calibration image
# will be picked for calibration, depending upon pattern
# extracted by cv2.findChessboardCorners, some calibrate
# image will yield less recognized corners than others..
# My list of empirical patternSize to calibrate camera using all
# provided calibration images. There is NO single pattern works
# for all 20 images, E.g. we have following special patternSize:
# N.B. image pattern method to pick correct dest. obj points
# calibration1: (9, 5) - Upper 5 rows, if mean(Y) < ( 720/2)
# calibration4: (5, 6) - Left 5 columns, if mean(X) < (1280/2)
# calibration5: (7, 6) - right 7 columns, if mean(X) > (1280/2)
# other images: (9, 6) - pick all
# So try vectors: (9, 6), (9, 5), (7, 6), (5, 6)
# In the implementation method below I iterate through different
# patternSize, and populate correct objectpoints automatically!
patternSz = [(9, 6), (9, 5), (7, 6), (5, 6)]
# List of to be calibrated images
images = glob.glob('camera_cal/calibration*.jpg')
imgNum = len(images)
# List of buffers to store and display calibrated images
# There are imgNum (20) calibrated images
Image = np.zeros((imgNum, h, w, 3), np.uint8)
# Global Flags
CamCalibrated = False
# Camera matrix and distortion coefficients are produced later
#################################################################
# Perspective Tranform section (most not used in video pipeline)
#################################################################
# List of test images in '/test_images'
timages = glob.glob('test_images/*.jpg')
timgNum = len(timages)
# Placeholder for undistorted test images
uImage = np.zeros((timgNum, h, w, 3), np.uint8)
# Placeholder for Perspective Transformed (birdeye) test images
tImage = np.zeros((timgNum, h, w, 3), np.uint8)
#############################################################################
# Color space and Gradient/Sobel Filter section (not used in video pipeline)
#############################################################################
# Placeholder for Filtered (S-channel and Sobel-x combined) test images
fImage = np.zeros((timgNum, h, w), np.uint8)
##############################
# Lane Lines Finding section
##############################
# Define y-value where we want radius of curvature
# Choose the maximum y-value, corresponding to the bottom of the image
y_eval = h # h = 720
# Definition for polynomial fitted line points
# fitted line points Y values
yvals = np.arange(h)
# fitted line X values for test image
tl_fitx = np.zeros((timgNum, h))
tr_fitx = np.zeros((timgNum, h))
# Lane line Curvature Radius (m) of test images
rad = np.zeros((timgNum))
# Lane line Center Departure (cm) of test images
dev = np.zeros((timgNum))
# Define conversions in x and y from pixels space to meters
# Warning: These numbers aren't calibrated according to our
# region selection for perspective transformations
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
# Placeholder for final Warped back (onto undistorted vs. very original) test images
wImage = np.zeros((timgNum, h, w), np.uint8)
# Return (retval, mtx, dist)
def cameraCalibrate(imagelist):
global CamCalibrated
if CamCalibrated == True:
return (0,0,0)
# Arrays to store object points and image points from all calibration images
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane
# Step through the list and search for chessboard corners
for idx, fname in enumerate(imagelist):
img = cv2.imread(fname)
# Normalize image size. N.B. calibration7/15 shape = (721, 1281, 3)
img = img[0:h,0:w,:]
# output image with ChessboardCorners Drew
outfile = outputpath + fname.split('/')[1].split('.')[0] + '-corners.jpg'
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
for pattern in patternSz:
# Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, pattern, None)
# If found, add object points, image points
if ret == True:
# Calc geometry center of found corners
# Use them to identify object points in
# closeup shots (i.e. not all included)
extracted = corners.reshape(-1, 2)
geoCenter = np.mean(extracted, axis=0)
# in case (9, 5) select correct rows
if pattern == patternSz[1]:
if geoCenter[1] < h/2:
# top 5 rows of objp, use list comprehension
rows= [i for i in range(objp.shape[0] - 9)]
else:
# bottom 5 rows objp
rows= [i for i in range(9, objp.shape[0])]
objpoints.append(objp[rows,:])
imgpoints.append(corners)
# in case (7, 6) select correct columns
elif pattern == patternSz[2]:
if geoCenter[0] < w/2:
# left 7 columns of objp, use list comprehension
rows= [i for i in range(objp.shape[0]) if (i%9) < 7]
else:
# right 7 columns objp, to pick correct rows from 1D of 6*9
rows= [i for i in range(objp.shape[0]) if (i%9) > 1]
objpoints.append(objp[rows,:])
imgpoints.append(corners)
# in case (5, 6) select correct columns
elif pattern == patternSz[3]:
if geoCenter[0] < w/2:
# left 5 columns of objp, use list comprehension
rows= [i for i in range(objp.shape[0]) if (i%9) < 5]
else:
# right 5 columns objp, to pick correct rows from 1D of 6*9
rows= [i for i in range(objp.shape[0]) if (i%9) > 3]
objpoints.append(objp[rows,:])
imgpoints.append(corners)
# in case (9, 6) select all objp points as default
else:
objpoints.append(objp)
imgpoints.append(corners)
# Draw and display the corners
cv2.drawChessboardCorners(img, pattern, corners, ret)
Image[idx] = img
cv2.imwrite(outfile, cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
break
retval, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, imsize,None,None)
# We won't use rvecs, tvecs in this project
# calibrateCamera returns the root mean square (RMS) re-projection error,
# usually it should be between 0.1 and 1.0 pixels in a good calibration.
# An RMS error of 1.0 means on average each of these projected points is
# 1.0 px away from its actual position.
# Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
dist_pickle = {}
dist_pickle["mtx"] = mtx
dist_pickle["dist"] = dist
pickle.dump( dist_pickle, open( "camera_cal/distpickle.p", "wb" ) )
CamCalibrated = True
return (retval, mtx, dist)
mtx) and Distortion Coefficients (dist) to be used in Pipeline¶# This cell generates global camera matrix and distortion coefficients that are used in image pipeline
# Calibrate Camera:
# Generate -OR- Load Global Camera Matrix, Distortion Coefficients here!
rms, mtx, dist = cameraCalibrate(images)
print(rms)
if (rms, mtx, dist) == (0,0,0):
# then load camera and distrortion matrix from pre-pickled file: `camera_cal/distpickle.p`
dist_file = 'camera_cal/distpickle.p'
with open(dist_file, mode='rb') as f:
dist_pickle = pickle.load(f)
mtx = dist_pickle['mtx']
dist = dist_pickle['dist']
output_images/*-corners.jpg)¶plot_gallery(Image, images, 720, 1280, 5, 4)
output_images/*-undist.jpg)¶for idx, fname in enumerate(images):
img = cv2.imread(fname)
img = img[0:h,0:w,:]
# output undistorted image
outfile = outputpath + fname.split('/')[1].split('.')[0] + '-undist.jpg'
dst = cv2.undistort(img, mtx, dist, None, mtx)
Image[idx] = dst
cv2.imwrite(outfile, cv2.cvtColor(dst, cv2.COLOR_BGR2RGB))
plot_gallery(Image, images, 720, 1280, 5, 4)
# This cell generates global perspective tranform (and Inverse) matrix that are used in image pipeline
# It's not part of direct image pipeline, rather than a parameter tunning (M & Minv) and demonstration
# consider challenge (forest) video, it is suggested that we use parameterized field depth vs. static
# In other words, use much shorter road distance (10-20m vs. 30m). Lane width may still be 3.7m
## TODO: Set following values - less hard code please
src = np.float32([[530, 480], [ 770, 480], [50, 720], [1250, 720]])
dst = np.float32([[ 50, 0], [1250, 0], [50, 720], [1250, 720]])
M = cv2.getPerspectiveTransform(src, dst)
Minv = cv2.getPerspectiveTransform(dst, src)
# N.B. So we have acquired perspective transform matrix M, but it is rather static
# we don't have to generate it from each frame in video pipeline, just use it
# CONS: Limitation comes from this static setting (src/dst points selection): it is
# expected that when road is narrower, windier or hillier, It may fail!
# TODO: Find better/adaptive perspective transformation scheme for challenge videos
#######################################################################################
# Steps below show undistorted and perspective transformed birdeye views on test images
#
# N.B.
# And validated (visually) correct parameter setting (src & dst points) for transform
#######################################################################################
# Step through the test images with birdeye images generated, plot them side-by-side
# to assess the `correct` setting of our PerspectiveTransform (given selected points).
for idx, fname in enumerate(timages):
# output birdeye image
outfile = outputpath + fname.split('/')[1].split('.')[0] + '-birdeye.jpg'
img = cv2.imread(fname)
# this cv2.resize is kept here to deal with
# `solidWhiteRight.jpg` and `solidYellowLeft.jpg`
if img.shape[:2] != (720, 1280):
img = cv2.resize(img, (1280, 720), interpolation = cv2.INTER_CUBIC)
# undistort image from prior calibrations:
img = cv2.undistort(img, mtx, dist, None, mtx)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
uImage[idx] = img
birdeye = cv2.warpPerspective(img, M, imsize, flags=cv2.INTER_LINEAR)
tImage[idx] = birdeye
cv2.imwrite(outfile, cv2.cvtColor(birdeye, cv2.COLOR_BGR2RGB))
# Mid-x = 650 vs. 640, use 650 as lane center
# Draw Region of Selection points for PerspectiveTransform on image
# N.B. This is just for demonstration of Region of Selection. These
# lines should NOT be drawn on images to polyfit() in Pipeline
## pts = np.array([[530,480],[770,480],[1250,720],[50,720]], np.int32)
## TODO
pts = np.array([[530,480],[770,480],[1250,720],[50,720]], np.int32)
pts = pts.reshape((-1,1,2))
cv2.polylines(img,[pts],True,(0,255,255),1)
##############################################################################
# Drew (in Left column images) Region of Selection `src` for birdeye transform
##############################################################################
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img)
ax1.set_title('Undistort: ' + fname[12:], fontsize=30)
ax2.imshow(birdeye)
ax2.set_title('Bird-Eye View: ' + fname[12:], fontsize=30)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)